#include <iostream>
#include <string>
#include <vector>
#include <boost/filesystem.hpp>
#include "util.hpp"
//html文件存放目录及数据清洗后的存放位置
const std::string src_path = "data/input";
const std::string output = "data/raw_html/raw.txt";

typedef struct DocInfo
{
  std::string title;    // 标题
  std::string content;  // 内容
  std::string url;      // 网址
}DocInfo_t;

bool EnumFile(const std::string &src_path,std::vector<std::string>* files_list);
bool ParseHtml(const std::vector<std::string>& files_list,std::vector<DocInfo_t>* results);
bool SaveHtml(const std::vector<DocInfo_t>& results,const std::string& output);

int main()
{
  std::vector<std::string> files_list;
  //1.递归式地把每个html文件名和路径保存到files_list中
  if(!EnumFile(src_path,&files_list))
  {
    std::cerr<<"enum file name fail!"<<std::endl;
    return 1;
  }
  //2.按照file_list读取文件内容并解析
  std::vector<DocInfo_t> results;
  if(!ParseHtml(files_list,&results))
  {
    std::cerr<<"Parse html fail!"<<std::endl;
    return 2;
  }
  //3.把解析完毕的各个文件内容写入到output文件，以/3为分隔符
  if(!SaveHtml(results,output))
  {
    std::cerr<<"Save html fail!"<<std::endl;
    return 3;
  }
}

bool EnumFile(const std::string &src_path,std::vector<std::string>* files_list)
{
  namespace fs = boost::filesystem;
  fs::path root_path(src_path);
  if(!fs::exists(root_path))
  {
    std::cerr<<src_path<<"not exits!"<<std::endl;
    return false;
  }
  //定义一个空的迭代器用来判断递归结束
  fs::recursive_directory_iterator end;
  for(fs::recursive_directory_iterator iter(root_path);iter!= end;iter++)
  {
    //判断是否是普通文件
    if(!fs::is_regular_file(*iter))
    {
      continue;
    }
    //判断文件后缀
    if(iter->path().extension()!=".html")
    {
      continue;
    }
    //std::cout<<"Debug:"<<iter->path().string()<<std::endl;
    files_list->push_back(iter->path().string());
  }
  return true;
}

static bool ParseTitle(const std::string& file,std::string* title)
{
  std::size_t begin = file.find("<title>");
  if(begin == std::string::npos)
  {
    return false;
  }
  std::size_t end = file.find("</title>");
  if(end == std::string::npos)
  {
    return false;
  }
  begin += std::string("<title>").size();
  if(begin > end)
  {
    return false;
  }
  *title = file.substr(begin,end-begin);
  return true;
}
static bool ParseContent(const std::string& file,std::string* content)
{
  enum status
  {
    LABLE,
    CONTENT 
  };
  enum status s = LABLE;
  for(char c: file)
  {  
    switch(s)
    {
      case LABLE:
        if(c=='>') s=CONTENT;
        break;
      case CONTENT:
        if(c == '<') s = LABLE;
        else 
        {
          if(c =='\n') c = ' ';
          content->push_back(c);
        }
        break;
      default:
        break;
    }
  }
  return true;
}
static bool ParseUrl(const std::string& file_path,std::string* url)
{
  std::string url_head = "https://www.boost.org/doc/libs/1_79_0/doc/html";
  std::string url_tail = file_path.substr(src_path.size());
  *url = url_head + url_tail;
  return true;
}
static void ShowDock(const DocInfo_t& doc)
{
  std::cout<<"title:"<<doc.title<<std::endl;
  std::cout<<"content:"<<doc.content<<std::endl;
  std::cout<<"url:"<<doc.url<<std::endl;
}
bool ParseHtml(const std::vector<std::string>& files_list,std::vector<DocInfo_t>* results)
{
  for(const std::string& file :files_list)
  {
    //读文件
    std::string result;
    if(!ns_util::FileUtil::ReadFile(file,&result))
    {
      continue;
    }
    DocInfo_t doc;
    //提取title
    if(!ParseTitle(result,&doc.title))
    {
      continue;
    }
    //提取content
    if(!ParseContent(result,&doc.content))
    {
      continue;
    }
    //构建url
    if(!ParseUrl(file,&doc.url))
    {
      continue;
    }
    //移动构造减少拷贝
    results->push_back(std::move(doc));

    //for debug
    //ShowDock(doc);
    //break;
  }
  return true;
}
bool SaveHtml(const std::vector<DocInfo_t>& results,const std::string& output)
{
#define SEP '\3'
  std::ofstream out(output , std::ios::out | std::ios::binary);
  if(!out.is_open())
  {
    std::cerr<<"open "<<output<<" failed!"<<std::endl;
    return false;
  }
  for(auto item : results)
  {
    std::string out_string;
    out_string += item.title;
    out_string += SEP;
    out_string += item.content;
    out_string += SEP;
    out_string += item.url;
    out_string += '\n';

    out.write(out_string.c_str(),out_string.size());
  }
  out.close();
  return true;
}
